{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://github.com/NTMC-Community/MatchZoo/blob/2.0/artworks/matchzoo-logo.png?raw=True\" alt=\"logo\" style=\"width:600px;float: center\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:27.539731Z",
     "start_time": "2019-01-11T16:11:24.900110Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.062500Z",
     "start_time": "2019-01-11T16:11:27.541704Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_pack = mz.datasets.wiki_qa.load_data(stage='train', task='ranking')\n",
    "test_data_pack = mz.datasets.wiki_qa.load_data(stage='test', task='ranking')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.072297Z",
     "start_time": "2019-01-11T16:11:28.064116Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matchzoo.data_pack.data_pack.DataPack"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_data_pack)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`DataPack` is a MatchZoo native data structure that most MatchZoo data handling processes build upon. A `DataPack` is consists of three `pandas.DataFrame`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.092391Z",
     "start_time": "2019-01-11T16:11:28.076906Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q1</th>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q5</th>\n",
       "      <td>how did apollo creed die</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q6</th>\n",
       "      <td>how long is the term for federal judges</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q7</th>\n",
       "      <td>how a beretta model 21 pistols magazines works</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q1                           how are glacier caves formed?\n",
       "Q2       How are the directions of the velocity and for...\n",
       "Q5                                how did apollo creed die\n",
       "Q6                 how long is the term for federal judges\n",
       "Q7          how a beretta model 21 pistols magazines works"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_pack.left.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.102613Z",
     "start_time": "2019-01-11T16:11:28.095519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1-0</th>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-1</th>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-2</th>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-3</th>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-4</th>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_right\n",
       "id_right                                                   \n",
       "D1-0      A partly submerged glacier cave on Perito More...\n",
       "D1-1              The ice facade is approximately 60 m high\n",
       "D1-2              Ice formations in the Titlis glacier cave\n",
       "D1-3      A glacier cave is a cave formed within the ice...\n",
       "D1-4      Glacier caves are often called ice caves , but..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_pack.right.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.110804Z",
     "start_time": "2019-01-11T16:11:28.104140Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left id_right  label\n",
       "0      Q1     D1-0      0\n",
       "1      Q1     D1-1      0\n",
       "2      Q1     D1-2      0\n",
       "3      Q1     D1-3      1\n",
       "4      Q1     D1-4      0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_pack.relation.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to convert a `DataPack` into a single `pandas.DataFrame` that holds all information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.248938Z",
     "start_time": "2019-01-11T16:11:28.112696Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...      0  \n",
       "1          The ice facade is approximately 60 m high      0  \n",
       "2          Ice formations in the Titlis glacier cave      0  \n",
       "3  A glacier cave is a cave formed within the ice...      1  \n",
       "4  Glacier caves are often called ice caves , but...      0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_pack.frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, using such `pandas.DataFrame` consumes much more memory if there are many duplicates in the texts, and that is the exact reason why we use `DataPack`. For more details about data handling, consult `matchzoo/tutorials/data_handling.ipynb`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MatchZoo preprocessors are used to convert a raw `DataPack` into a `DataPack` that ready to be fed into a model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:28.252857Z",
     "start_time": "2019-01-11T16:11:28.250331Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.preprocessors.NaivePreprocessor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are two steps to use a preprocessor. First, `fit`. Then, `transform`. `fit` will only changes the preprocessor's inner state but not the input `DataPack`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:32.158937Z",
     "start_time": "2019-01-11T16:11:28.254265Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 2118/2118 [00:00<00:00, 9331.92it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 18841/18841 [00:03<00:00, 5463.60it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 789717.83it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 737564.58it/s]\n",
      "Building VocabularyUnit from a datapack.: 100%|██████████| 418401/418401 [00:00<00:00, 2731397.41it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matchzoo.preprocessors.naive_preprocessor.NaivePreprocessor at 0x10b6c51d0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.fit(train_data_pack)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`fit` will gather all information it needs into its `context`. In the above example, we can see a `VocabularyUnit` is built during the fitting process using `train_data_pack`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:32.165211Z",
     "start_time": "2019-01-11T16:11:32.160845Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vocab_unit': <matchzoo.processor_units.processor_units.VocabularyUnit at 0x10c44a470>}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`VocabularyUnit` is a `StatefulProcessorUnit` that has a similar `fit`/`transform` interface. Once a `VocabularyUnit` `fit`, it will store a mapping from `term` to `index` and the reverse in its `state`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `NaivePreprocessor` already handles `VocabularyUnit` internally, so we do not have to worry about that. Just access it through the `NaivePreprocessor`'s `context`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:32.174447Z",
     "start_time": "2019-01-11T16:11:32.168007Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6363\n",
      "15319\n",
      "coinage\n",
      "song\n"
     ]
    }
   ],
   "source": [
    "vocab_unit = preprocessor.context['vocab_unit']\n",
    "print(vocab_unit.state['term_index']['match'])\n",
    "print(vocab_unit.state['term_index']['zoo'])\n",
    "print(vocab_unit.state['index_term'][1])\n",
    "print(vocab_unit.state['index_term'][2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once `fit`, the preprocessor has enough information to `transform`.  `transform` will not change the preprocessor's inner state and the input `DataPack`, but return a transformed `DataPack`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.665791Z",
     "start_time": "2019-01-11T16:11:32.176291Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 2118/2118 [00:00<00:00, 7925.25it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 18841/18841 [00:03<00:00, 4827.49it/s]\n",
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 633/633 [00:00<00:00, 9211.28it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 5961/5961 [00:01<00:00, 4832.43it/s]\n"
     ]
    }
   ],
   "source": [
    "train_data_pack_processed = preprocessor.transform(train_data_pack)\n",
    "test_data_pack_processed = preprocessor.transform(test_data_pack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.675698Z",
     "start_time": "2019-01-11T16:11:37.667459Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q1</th>\n",
       "      <td>[27331, 16812, 9507, 13770, 14516, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>[27331, 16812, 5253, 21768, 28148, 5253, 27245...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q5</th>\n",
       "      <td>[27331, 21186, 3354, 29843, 10061, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q6</th>\n",
       "      <td>[27331, 25346, 23867, 5253, 13066, 3895, 13948...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q7</th>\n",
       "      <td>[27331, 21700, 28244, 5424, 24372, 29157, 7952...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q1       [27331, 16812, 9507, 13770, 14516, 0, 0, 0, 0,...\n",
       "Q2       [27331, 16812, 5253, 21768, 28148, 5253, 27245...\n",
       "Q5       [27331, 21186, 3354, 29843, 10061, 0, 0, 0, 0,...\n",
       "Q6       [27331, 25346, 23867, 5253, 13066, 3895, 13948...\n",
       "Q7       [27331, 21700, 28244, 5424, 24372, 29157, 7952..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_pack_processed.left.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, `text_left` is already in sequence form that nerual networks love."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just to make sure we have the correct sequence:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.681721Z",
     "start_time": "2019-01-11T16:11:37.677406Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before: how are glacier caves formed?\n",
      "After: [27331, 16812, 9507, 13770, 14516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "Translated: how_are_glacier_caves_formed_________________________\n"
     ]
    }
   ],
   "source": [
    "print('Before:', train_data_pack.left.loc['Q1']['text_left'])\n",
    "sequence = train_data_pack_processed.left.loc['Q1']['text_left']\n",
    "print('After:', sequence)\n",
    "print('Translated:', '_'.join([vocab_unit.state['index_term'][i] for i in sequence]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more details about preprocessing, consult `matchzoo/tutorials/data_handling.ipynb`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MatchZoo provides many built-in text matching models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.688429Z",
     "start_time": "2019-01-11T16:11:37.683627Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.models.naive.Naive,\n",
       " matchzoo.models.dssm.DSSM,\n",
       " matchzoo.models.cdssm.CDSSM,\n",
       " matchzoo.models.dense_baseline.DenseBaseline,\n",
       " matchzoo.models.arci.ArcI,\n",
       " matchzoo.models.arcii.ArcII,\n",
       " matchzoo.models.match_pyramid.MatchPyramid,\n",
       " matchzoo.models.knrm.KNRM,\n",
       " matchzoo.models.duet.DUET,\n",
       " matchzoo.models.drmmtks.DRMMTKS,\n",
       " matchzoo.models.drmm.DRMM,\n",
       " matchzoo.models.anmm.ANMM,\n",
       " matchzoo.models.mvlstm.MVLSTM,\n",
       " matchzoo.contrib.models.match_lstm.MatchLSTM]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.models.list_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.692005Z",
     "start_time": "2019-01-11T16:11:37.689803Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = mz.models.DenseBaseline()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model is initialized with a hyper parameter table, in which values are partially filled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.699457Z",
     "start_time": "2019-01-11T16:11:37.694266Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name                          None\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  None\n",
      "task                          None\n",
      "optimizer                     None\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 256\n",
      "mlp_num_layers                None\n",
      "mlp_num_fan_out               None\n",
      "mlp_activation_func           None\n"
     ]
    }
   ],
   "source": [
    "print(model.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.712969Z",
     "start_time": "2019-01-11T16:11:37.708767Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name                          My First Model\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  None\n",
      "task                          None\n",
      "optimizer                     None\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 3\n",
      "mlp_num_layers                None\n",
      "mlp_num_fan_out               None\n",
      "mlp_activation_func           None\n"
     ]
    }
   ],
   "source": [
    "model.params['name'] = 'My First Model'\n",
    "model.params['mlp_num_units'] = 3\n",
    "print(model.params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `guess_and_fill_missing_params` to automatically fill-in other hyper parameters. This involves some guessing so the parameter it fills could be wrong. For example, the default task is `Ranking`, and if we do not set it to `Classification` manaully for data packs prepared for classification, then the shape of the model output and the data will mismatch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.721827Z",
     "start_time": "2019-01-11T16:11:37.718542Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter \"task\" set to Ranking Task.\n",
      "Parameter \"input_shapes\" set to [(30,), (30,)].\n",
      "Parameter \"optimizer\" set to adam.\n",
      "Parameter \"mlp_num_layers\" set to 3.\n",
      "Parameter \"mlp_num_fan_out\" set to 32.\n",
      "Parameter \"mlp_activation_func\" set to relu.\n",
      "name                          My First Model\n",
      "model_class                   <class 'matchzoo.models.dense_baseline.DenseBaseline'>\n",
      "input_shapes                  [(30,), (30,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_multi_layer_perceptron   True\n",
      "mlp_num_units                 3\n",
      "mlp_num_layers                3\n",
      "mlp_num_fan_out               32\n",
      "mlp_activation_func           relu\n"
     ]
    }
   ],
   "source": [
    "model.guess_and_fill_missing_params()\n",
    "print(model.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.727515Z",
     "start_time": "2019-01-11T16:11:37.723686Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.params.completed()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With all parameters filled in, we can now build and compile the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:37.820447Z",
     "start_time": "2019-01-11T16:11:37.729860Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 30)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 30)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 60)           0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 3)            183         concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 3)            12          dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 3)            12          dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 32)           128         dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            33          dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 368\n",
      "Trainable params: 368\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-10T13:28:54.044570Z",
     "start_time": "2018-12-10T13:28:54.038325Z"
    }
   },
   "source": [
    "For more details about models, consult `matchzoo/tutorials/models.ipynb`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train, Evaluate, Predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A `DataPack` can `unpack` itself into data that can be directly used to train a MatchZoo model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:38.093053Z",
     "start_time": "2019-01-11T16:11:37.823282Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, y = train_data_pack_processed.unpack()\n",
    "test_x, test_y = test_data_pack_processed.unpack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:42.173960Z",
     "start_time": "2019-01-11T16:11:38.094875Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "20360/20360 [==============================] - 1s 49us/step - loss: 181925.8526\n",
      "Epoch 2/5\n",
      "20360/20360 [==============================] - 1s 32us/step - loss: 51.1456\n",
      "Epoch 3/5\n",
      "20360/20360 [==============================] - 1s 34us/step - loss: 1.6366\n",
      "Epoch 4/5\n",
      "20360/20360 [==============================] - 1s 33us/step - loss: 1.4964\n",
      "Epoch 5/5\n",
      "20360/20360 [==============================] - 1s 34us/step - loss: 1.4385\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1393d1780>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x, y, batch_size=32, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An alternative to train a model is to use a `DataGenerator`. This might be useful for delaying expensive preprocessing steps or doing real-time data augmentation. For more details about `DataGenerator`, consult `matchzoo/tutorials/data_handling.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:42.181904Z",
     "start_time": "2019-01-11T16:11:42.175798Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_generator = mz.DataGenerator(train_data_pack_processed, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:55.810000Z",
     "start_time": "2019-01-11T16:11:42.185169Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "637/637 [==============================] - 2s 4ms/step - loss: 1.3603\n",
      "Epoch 2/5\n",
      "637/637 [==============================] - 3s 4ms/step - loss: 1.2706\n",
      "Epoch 3/5\n",
      "637/637 [==============================] - 3s 4ms/step - loss: 1.1472\n",
      "Epoch 4/5\n",
      "637/637 [==============================] - 3s 5ms/step - loss: 1.0140\n",
      "Epoch 5/5\n",
      "637/637 [==============================] - 3s 4ms/step - loss: 0.8714\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x13934c6d8>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(data_generator, epochs=5, use_multiprocessing=True, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:55.960804Z",
     "start_time": "2019-01-11T16:11:55.814436Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mae': array([0.38911337, 0.31755346, 0.20464206, ..., 0.20464206, 0.20464206,\n",
       "        0.20464206], dtype=float32)}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_x, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:11:55.998007Z",
     "start_time": "2019-01-11T16:11:55.962804Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.38911337],\n",
       "       [0.31755346],\n",
       "       [0.20464206],\n",
       "       ...,\n",
       "       [0.20464206],\n",
       "       [0.20464206],\n",
       "       [0.20464206]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(test_x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-10T14:00:25.446639Z",
     "start_time": "2018-12-10T14:00:25.444180Z"
    }
   },
   "source": [
    "# Automation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MatchZoo strives for ease of use, and package `matchzoo.auto` is a perfect example of that."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`matchzoo.auto.prepare` handles interaction among data, model, and preprocessor automatically. For example, some model like `DSSM` have dynamic input shapes based on the result of word hashing. Some models have an embedding layer which dimension is related to the data's vocabulary size. `prepare` takes care of all that and returns properly prepared model, data, and preprocessor for you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:12:00.277133Z",
     "start_time": "2019-01-11T16:11:56.006960Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => StopRemovalUnit => NgramLetterUnit: 100%|██████████| 13/13 [00:00<00:00, 2205.30it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => StopRemovalUnit => NgramLetterUnit: 100%|██████████| 100/100 [00:00<00:00, 2453.57it/s]\n",
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 11431.02it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 89526.23it/s]\n",
      "Building VocabularyUnit from a datapack.: 100%|██████████| 8523/8523 [00:00<00:00, 2096782.98it/s]\n",
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => StopRemovalUnit => NgramLetterUnit => WordHashingUnit: 100%|██████████| 13/13 [00:00<00:00, 2334.86it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => StopRemovalUnit => NgramLetterUnit => WordHashingUnit: 100%|██████████| 100/100 [00:00<00:00, 2373.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter \"name\" set to DSSM.\n",
      "Parameter \"mlp_num_layers\" set to 3.\n",
      "Parameter \"mlp_num_units\" set to 64.\n",
      "Parameter \"mlp_num_fan_out\" set to 32.\n",
      "Parameter \"mlp_activation_func\" set to relu.\n",
      "Epoch 1/1\n",
      "100/100 [==============================] - 1s 5ms/step - loss: 0.0979\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mae': array([0.3854349 , 0.2650984 , 0.21409294, ..., 0.02204863, 0.01172964,\n",
       "        0.03805733], dtype=float32)}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_ok, train_ok, preprocesor_ok = mz.auto.prepare(\n",
    "    model=mz.models.DSSM(),\n",
    "    data_pack=train_data_pack[:100]\n",
    ")\n",
    "test_ok = preprocesor_ok.transform(test_data_pack, verbose=0)\n",
    "model_ok.fit(*train_ok.unpack(), batch_size=32)\n",
    "model_ok.evaluate(*test_ok.unpack())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more details about automation, consult `matchzoo/tutorials/automation.ipynb`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-12-21T14:38:51.631Z"
    }
   },
   "source": [
    "# Full Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:12:00.284468Z",
     "start_time": "2019-01-11T16:12:00.279165Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_classes = [\n",
    "    mz.models.CDSSM,\n",
    "    mz.models.DSSM,\n",
    "    mz.models.DUET,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:12:58.119596Z",
     "start_time": "2019-01-11T16:12:00.294428Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "task = mz.tasks.Ranking(metrics=['ap', 'ndcg'])\n",
    "results = []\n",
    "for model_class in model_classes:\n",
    "    print(model_class)\n",
    "    model = model_class()\n",
    "    model.params['task'] = task\n",
    "    model_ok, train_ok, preprocesor_ok = mz.auto.prepare(\n",
    "        model=model,\n",
    "        data_pack=train_data_pack[:2000],\n",
    "        verbose=0\n",
    "    )\n",
    "    test_ok = preprocesor_ok.transform(test_data_pack, verbose=0)\n",
    "    callback = mz.engine.callbacks.EvaluateAllMetrics(\n",
    "        model_ok,\n",
    "        *test_ok.unpack(),\n",
    "        batch_size=1024,\n",
    "        verbose=0\n",
    "    )\n",
    "    history = model_ok.fit(*train_ok.unpack(), batch_size=32, epochs=3, callbacks=[callback])\n",
    "    results.append({'name': model_ok.params['name'], 'history': history})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:12:58.138199Z",
     "start_time": "2019-01-11T16:11:25.113Z"
    }
   },
   "outputs": [],
   "source": [
    "import bokeh\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.io import export_png\n",
    "from bokeh.layouts import column\n",
    "from bokeh.models.tools import HoverTool\n",
    "import IPython\n",
    "\n",
    "charts = {\n",
    "    metric: figure(\n",
    "        title=str(metric),\n",
    "        sizing_mode='scale_width',\n",
    "        width=800, height=400\n",
    "    ) for metric in results[0]['history'].history.keys()\n",
    "}\n",
    "hover_tool = HoverTool(tooltips=[\n",
    "    (\"x\", \"$x\"),\n",
    "    (\"y\", \"$y\")\n",
    "])\n",
    "for metric, sub_chart in charts.items():\n",
    "    lines = {}\n",
    "    for result, color in zip(results, bokeh.palettes.Category10[10]):\n",
    "        x = result['history'].epoch\n",
    "        y = result['history'].history[metric]\n",
    "        lines[result['name']] = sub_chart.line(\n",
    "            x, y, color=color, line_width=2, alpha=0.5, legend=result['name'])\n",
    "        sub_chart.add_tools(hover_tool)\n",
    "\n",
    "export_png(column(*charts.values()), \"quick_start_chart.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-24T06:36:42.718333Z",
     "start_time": "2018-12-24T06:36:42.590961Z"
    },
    "scrolled": false
   },
   "source": [
    "![chart](./quick_start_chart.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "matchzoo",
   "language": "python",
   "name": "matchzoo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
